#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          to be called from test21.30
#          Tests if the group nodes limit is enforced
############################################################################
# Copyright (C) 2012 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

# Test GrpNode limit
proc inc21_30_1 { wait_reason } {
	global srun salloc acct bin_sleep number grn_num check_reason

	log_info "Starting GrpNode limit test"
	set job_id1  0
	set job_id2  0

	# TODO Temporarily increase logging to debug bug 12658 (remove once fixed)
	global bin_echo test_id
	if {$test_id eq "21.30"} {
		set config_dir [get_conf_path]
		set config_file $config_dir/slurm.conf
		save_conf $config_file
		run_command -none "$bin_echo SlurmctldDebug=debug3 >> $config_file"
		run_command -none "$bin_echo DebugFlags=TraceJobs >> $config_file"
		reconfigure -fail
	}

	# Raise an error to abort the catch block
	set exception_code [catch {

		spawn $salloc -N$grn_num --account=$acct --exclusive -t1 $srun $bin_sleep 10
			expect {
				-re "Granted job allocation ($number)" {
					set job_id1 $expect_out(1,string)
					log_debug "Job $job_id1 has been submitted"
				}
				-re "Pending job allocation ($number)" {
					set job_id1 $expect_out(1,string)

					# TODO Temporary debug for bug 12658 (remove once fixed)
					global scontrol sacctmgr
					run_command "$scontrol show jobs"
					run_command "$scontrol show nodes"
					run_command "$scontrol show reservations"
					run_command "$sacctmgr show assoc"

					error "Job should be running, but is not"
				}
				-re "error" {
					error "Job allocation should not have failed"
				}
				timeout {
					error "salloc is not reponding"
				}
				eof {
					wait
				}
			}
		if {$job_id1 == 0} {
			error "Jobs were not submitted"
		}

		spawn $salloc -N1 --account=$acct --exclusive -t1 $srun $bin_sleep 10
			expect {
				-re "Pending job allocation ($number)" {
					set job_id2 $expect_out(1,string)
					log_debug "Job $job_id2 is waiting for resources. This is expected"
				}
				-re "Granted job allocation ($number)" {
					error "This should be pended but is not"
				}
				timeout {
					error "salloc is not reponding"
				}
				eof {
					wait
				}
			}
		if {$job_id2 == 0} {
			error "Jobs were not submitted"
		}

		subtest [check_job_state $job_id2 PENDING] "Job should be in the PENDING state"
		subtest [check_reason $job_id2 $wait_reason] "Job should have wait reason $wait_reason"

	} message] ; # Store the error message in $message

	# Cancels remaining jobs
	cancel_job $job_id1
	cancel_job $job_id2

	# TODO Temporarily restore logging to debug bug 12658 (remove once fixed)
	if {$test_id eq "21.30"} {
		restore_conf $config_file
		reconfigure
	}

	# Convert any errors into failures (after cleaning up)
	if {$exception_code == 1} { ; # errors only
	    fail "Failure testing GrpNode limit: $message"
	}
}
